# -*- coding: utf-8 -*-
import scrapy
import re
from scrapy.http import Request
from urllib import parse
from ArticleSpider.items import JobBoleArticleItem,ArticleItemLoader
from ArticleSpider.utils.common import get_md5
from scrapy.loader import ItemLoader
import sys


class JobboleSpider(scrapy.Spider):
    name = 'jobbole'
    allowed_domains = ['blog.jobbole.com']
   # start_urls = ['http://blog.jobbole.com/114391/']  #开始的url
    start_urls = ['http://blog.jobbole.com/all-posts/']  #开始的url



    def parse(self, response):
       # next_url = response.css(".next.page-numbers::attr(href)").extract()[0]
       # print("123123")
       # print(next_url)
       # sys.exit()
        post_nodes = response.css("#archive .floated-thumb .post-thumb a")
        # print("2222222222222222")
        for post_node in post_nodes :
            image_url = post_node.css("img::attr(src)").extract_first("")
            post_url = post_node.css("::attr(href)").extract_first("")
            yield  Request(url=parse.urljoin(response.url, post_url),meta={"front_image_url":image_url},callback=self.parse_detail)

        #提取下一页url并交给scrapy进行下载
        # next_url = response.css(".next.page-numbers::attr(href)").extract()[0]
        # if next_url:
        #     # print(next_url)
        #     # print('111111')
        #     yield Request(url=parse.urljoin(response.url,next_url),callback=self.parse)

     # 文章内页选择器
    def parse_detail(self, response):
        #css选择器 筛选
        front_image_url = response.meta.get("front_image_url","") #文章封面图
        #itemloader
        item_loader = ArticleItemLoader(item=JobBoleArticleItem(),response=response)
        item_loader.add_value("url_object_id",get_md5(response.url))
        item_loader.add_css("title",".entry-header h1::text")
        item_loader.add_value("url",response.url)
        item_loader.add_css("create_date",".entry-meta-hide-on-mobile::text")
        item_loader.add_value("front_image_url",[front_image_url])
        item_loader.add_css("praise_nums",".post-adds .hide-on-480::text")
        item_loader.add_css("comment_nums",".post-adds .hide-on-480::text")
        item_loader.add_css("fav_nums",".bookmark-btn::text")
        item_loader.add_css("tags",".entry-meta-hide-on-mobile a::text")
        item_loader.add_css("content","div .entry")
        article_item = item_loader.load_item()
        yield article_item
        pass
